# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import os
import re
from html import unescape

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
今日头条-app
搜索内容
https://tsearch-quic.snssdk.com/search
正文内容
https://a3-ipv6.pstatp.com/article/content/24/1/{id}/{id}/0/0/1
评论内容
https://api100-quic-c.snssdk.com/article/v4/tab_comments
"""

class TouTiao():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            'Host': 'tsearch-quic.snssdk.com',
            'Connection': 'keep-alive',
            'Cookie': 'odin_tt=2d21e3bd96d90bbdcfb5d65d57243e9a19c2cf309f89ebabcc1dbbe5855ce8df3456d5f0316a92e6a04fbd5d0f49a681; install_id=2154687474445303; ttreq=1$f820528238516826b4466c2479a3d9b9f57e53bd; history=alrvlFic6pJZXJCTWBmSmZt6Ke5KcPpSz5LU3OLGy3EX1v3WOMDEfomBgcEBAAAA%2F%2F8%3D; qh[360]=1; WIN_WH=360_563',
            # 'X-SS-REQ-TICKET': '1590977462868',
            'passport-sdk-version': '15',
            'sdk-version': '2',
            'X-SS-DP': '13',
            'x-tt-trace-id': '00-6da583d609f95c3e13cbc6c715dd000d-6da583d609f95c3e-01',
            'User-Agent': 'com.ss.android.article.news/7740 (Linux; U; Android 5.0.2; zh_CN; CHE-TL00H; Build/HonorCHE-TL00H; Cronet/TTNetVersion:e4fc96b0 2020-03-30 QuicVersion:0144d358 2020-03-24)',
            'Accept-Encoding': 'gzip, deflate',
            'X-Gorgon': '0404b8b800019059ad35d67a3293c943b8e73e987e5949c273a4',
            # 'X-Khronos': '1590977462'

        }

    def get_contents(self, offset):
        postData = {
            'search_id':'',
            'from_search_subtab':'',
            'source':'input',
            'has_count':'',
            'offset':'0',
            'is_ttwebview':'0',
            'is_incognito':'0',
            'count':'10',
            'action_type':'input_keyword_search',
            'keyword_type':'',
            'from':'news',  # 频道:"news"资讯 "search_tab" 综合
            'search_position':'search_bar',  # 频道:"news"咨询 "search_bar"综合
            'format':'json',
            'is_from_native':'1',
            'keyword':'996',  # 搜索关键词
            'pd':'information',
            'search_load_id':'1',
            'qc_query':'',
            'iid':'2154687474445303',
            'device_id':'66937151804',
            'ac':'wifi',
            'mac_address':'48%3Adb%3A50%3Ab9%3A85%3A30',
            'channel':'baidu_0411',
            'aid':'13',
            'app_name':'news_article',
            'version_code':'774',
            'version_name':'7.7.4',
            'device_platform':'android',
            'ab_version':'1419049%2C668775%2C1529247%2C1190522%2C1157750%2C1419597%2C1629530%2C1469498%2C1484966%2C1576657%2C1593455%2C1742591%2C668779%2C662099%2C1403340%2C1706992%2C668774%2C1702947%2C660830%2C662176%2C1738571',
            'ab_feature':'102749%2C94563',
            'ssmix':'a',
            'device_type':'CHE-TL00H',
            'device_brand':'Honor',
            'language':'zh',
            'os_api':'21',
            'os_version':'5.0.2',
            'uuid':'869139022485730',
            'openudid':'a2bd3dc65bdf35cd',
            'manifest_version_code':'7740',
            'resolution':'720*1280',
            'dpi':'320',
            'update_version_code':'77410',
            '_rticket':int(time.time()*1000),
            'tma_jssdk_version':'1.65.0.5',
            'rom_version':'emotionui_3.1_che-tl00hc00b260-wanmei',
            'plugin':'0',
            'host_abi':'armeabi-v7a',
            'cdid':'8a761eee-dc56-4b3c-b1d6-117df15b910b'
        }
        url = "https://tsearch-quic.snssdk.com/search"
        time.sleep(3)
        html = HttpUtils.do_request("GET", url, self.headers, postData)

        # json_str = json.loads(html.text)['scripts'].replace('<script type="application/json" id="ala-data-6b0df5b0" data-for="ala-data" data-p="0">', '').replace()
        json_str = json.loads(html.text)['scripts']
        # json_str = re.findall(r"\{(.*?);", json_str)
        json_str = re.sub('<.*?>', '', json_str, flags=re.M | re.S)
        json_str = re.sub('window.T.*?;', ',', json_str, flags=re.M | re.S)
        json_str = "[" + json_str.strip()[:-1] + "]"
        data_json = json.loads(json_str)
        for d in data_json:
            dict_data = dict()
            id = d['id']
            dict_data['article_url'] = d['article_url']  # 文章链接
            dict_data['abstract'] = d['abstract']  # 摘要
            dict_data['title'] = d['title']  # 标题
            dict_data['media_name'] = d['media_name']  # 媒体名称
            dict_data['source'] = d['source']  # 作者
            dict_data['article_url'] = d['article_url']  # 文章链接
            dict_data['create_time'] = datetime.fromtimestamp(int(d['create_time']))\
                .strftime("%Y-%m-%d %H:%M:%S")  # 发布时间
            dict_data['comment_count'] = d['comment_count']  # 评论数量
            dict_data['digg_count'] = d['digg_count']  # 点赞数量
            dict_data['read_count'] = d['read_count']  # 阅读数量
            if d.get('user_auth_info') is not None:
                dict_data['auth_type'] = d['user_auth_info']['auth_type']  # 0 是V账号
            print(f"============={dict_data['title']}=============")
            # 判断日期范围
            date_start = datetime.strptime('2020-01-01', "%Y-%m-%d")
            date_end = datetime.strptime('2020-06-30', "%Y-%m-%d")
            date_now = datetime.strptime(dict_data['create_time'].split(" ")[0], "%Y-%m-%d")
            if date_now >= date_start and date_now <= date_end:
                # 正文内容
                self.get_contents_1(id, dict_data)
                # 评论内容
                self.get_comments(id, 0)

    # 正文内容
    def get_contents_1(self, id, dict_data):
        postData = {
            'iid':'2154687474445303',
            'device_id':'66937151804',
            'ac':'wifi',
            'mac_address':'48%3Adb%3A50%3Ab9%3A85%3A30',
            'channel':'baidu_0411',
            'aid':'13',
            'app_name':'news_article',
            'version_code':'774',
            'version_name':'7.7.4',
            'device_platform':'android',
            'ab_version':'1419049%2C668775%2C1529247%2C1190522%2C1157750%2C1419597%2C1629530%2C1469498%2C1484966%2C1576657%2C1593455%2C1742591%2C668779%2C662099%2C1403340%2C1706992%2C668774%2C1702947%2C660830%2C662176%2C1738571',
            'ab_feature':'102749%2C94563',
            'ssmix':'a',
            'device_type':'CHE-TL00H',
            'device_brand':'Honor',
            'language':'zh',
            'os_api':'21',
            'os_version':'5.0.2',
            'uuid':'869139022485730',
            'openudid':'a2bd3dc65bdf35cd',
            'manifest_version_code':'7740',
            'resolution':'720*1280',
            'dpi':'320',
            'update_version_code':'77410',
            '_rticket':int(time.time()*1000),
            'tma_jssdk_version':'1.65.0.5',
            'rom_version':'emotionui_3.1_che-tl00hc00b260-wanmei',
            'plugin':'0',
            'host_abi':'armeabi-v7a',
            'cdid':'8a761eee-dc56-4b3c-b1d6-117df15b910b'
        }
        url = f"https://a3-ipv6.pstatp.com/article/content/24/1/{id}/{id}/0/0/1"
        headers = {
            'Host': 'a3-ipv6.pstatp.com',
            'Connection': 'keep-alive',
            'X-SS-DP': '13',
            'x-tt-trace-id': '00-6d8af91409f95c3e13cb4d6736c3000d-6d8af91409f95c3e-01',
            'User-Agent': 'com.ss.android.article.news/7740 (Linux; U; Android 5.0.2; zh_CN; CHE-TL00H; Build/HonorCHE-TL00H; Cronet/TTNetVersion:e4fc96b0 2020-03-30 QuicVersion:0144d358 2020-03-24)',
            'Accept-Encoding': 'gzip, deflate',
            'X-Gorgon': '0404b8b80001f6f8c90fd67a320552abd1b63e987e59493e0669',
            # 'X-Khronos': '1590975723'

        }
        time.sleep(3)
        html = HttpUtils.do_request("GET", url, headers, postData)
        data_json = json.loads(html.text)
        # dict_data = dict()
        dict_data['content'] = data_json['data']['content']  # 内容
        with open("996.txt", "a+", encoding="utf-8") as fp:
            fp.write("--------------------正文内容--------------------" + '\n')

            fp.write("标题:" + dict_data['title'] + '\n')
            fp.write("作者:" + dict_data['source'] + '\n')
            fp.write("发布时间:" + dict_data['create_time'] + '\n')
            fp.write("评论数量:" + str(dict_data['comment_count']) + '\n')
            fp.write("正文内容:" + self.html_to_plain_text(dict_data['content']) + '\n')
        print("=============获取正文=============")
        # 评论内容
        self.get_comments(id, 0)


    # 评论内容
    def get_comments(self, id, pagenum):
        postData = {
            'count':'20',
            'category':'__search__',
            'comment_request_from':'1',
            'group_id':id,
            'offset':pagenum,
            'fold':'1',
            'iid':'2154687474445303',
            'device_id':'66937151804',
            'ac':'wifi',
            'mac_address':'48%3Adb%3A50%3Ab9%3A85%3A30',
            'channel':'baidu_0411',
            'aid':'13',
            'app_name':'news_article',
            'version_code':'774',
            'version_name':'7.7.4',
            'device_platform':'android',
            'ab_version':'668779%2C662099%2C1403340%2C1706992%2C668774%2C1702947%2C660830%2C662176%2C1738571%2C1419049%2C668775%2C1529247%2C1190522%2C1157750%2C1419597%2C1629530%2C1469498%2C1484966%2C1576657%2C1593455%2C1742591',
            'ab_feature':'102749%2C94563',
            'ssmix':'a',
            'device_type':'CHE-TL00H',
            'device_brand':'Honor',
            'language':'zh',
            'os_api':'21',
            'os_version':'5.0.2',
            'uuid':'869139022485730',
            'openudid':'a2bd3dc65bdf35cd',
            'manifest_version_code':'7740',
            'resolution':'720*1280',
            'dpi':'320',
            'update_version_code':'77410',
            '_rticket':'1590983334288',
            'tma_jssdk_version':'1.65.0.5',
            'rom_version':'emotionui_3.1_che-tl00hc00b260-wanmei',
            'plugin':'0',
            'host_abi':'armeabi-v7a&cdid:8a761eee-dc56-4b3c-b1d6-117df15b910b'
        }
        headers = {
            'Host': 'api100-quic-c.snssdk.com',
            'Connection': 'keep-alive',
            'Cookie': 'odin_tt=2d21e3bd96d90bbdcfb5d65d57243e9a19c2cf309f89ebabcc1dbbe5855ce8df3456d5f0316a92e6a04fbd5d0f49a681; install_id=2154687474445303; ttreq=1$f820528238516826b4466c2479a3d9b9f57e53bd; history=alrvlFic6pJZXJCTWBmSmZt6Ke5KcPpSz5LU3OLGy3EX1v3WOMDEfomBgcEBAAAA%2F%2F8%3D; qh[360]=1; WIN_WH=360_563',
            # 'X-SS-REQ-TICKET': '1590987800466',
            'passport-sdk-version': '15',
            'sdk-version': '2',
            'X-SS-DP': '13',
            'x-tt-trace-id': '00-6dff1b2409f95c3e13c4806969bc000d-6dff1b2409f95c3e-01',
            'User-Agent': 'com.ss.android.article.news/7740 (Linux; U; Android 5.0.2; zh_CN; CHE-TL00H; Build/HonorCHE-TL00H; Cronet/TTNetVersion:e4fc96b0 2020-03-30 QuicVersion:0144d358 2020-03-24)',
            'Accept-Encoding': 'gzip, deflate',
            # 'X-Gorgon': '0404b8b80001baae9479d67a3293c943b8e73e987e59495af270',
            # 'X-Khronos': '1590987800'
        }
        url = f'https://api100-quic-c.snssdk.com/article/v4/tab_comments'
        # url = f'https://api100-quic-c.snssdk.com/article/v4/tab_comments/?count=20&category=__search__&comment_request_from=1&group_id={id}&offset={pagenum}&fold=1&iid=2154687474445303&device_id=66937151804&ac=wifi&mac_address=48%3Adb%3A50%3Ab9%3A85%3A30&channel=baidu_0411&aid=13&app_name=news_article&version_code=774&version_name=7.7.4&device_platform=android&ab_version=668779%2C662099%2C1403340%2C1706992%2C668774%2C1702947%2C660830%2C662176%2C1738571%2C1419049%2C668775%2C1529247%2C1190522%2C1157750%2C1419597%2C1629530%2C1469498%2C1484966%2C1576657%2C1593455%2C1742591&ab_feature=102749%2C94563&ssmix=a&device_type=CHE-TL00H&device_brand=Honor&language=zh&os_api=21&os_version=5.0.2&uuid=869139022485730&openudid=a2bd3dc65bdf35cd&manifest_version_code=7740&resolution=720*1280&dpi=320&update_version_code=77410&_rticket=1590983334288&tma_jssdk_version=1.65.0.5&rom_version=emotionui_3.1_che-tl00hc00b260-wanmei&plugin=0&host_abi=armeabi-v7a&cdid=8a761eee-dc56-4b3c-b1d6-117df15b910b'
        html = HttpUtils.do_request("GET", url, headers, postData)
        # html = HttpUtils.do_request("GET", url, headers, "")
        time.sleep(3)
        if html.text != "":
            data_json = json.loads(html.text)
            counts = int(data_json['total_number'])
            if counts > 0:
                for d in data_json['data']:
                    dict_data = dict()
                    dict_data['user_name'] = d['comment']['user_name']  # 用户名称
                    dict_data['content'] = d['comment']['text']  # 评论内容
                    dict_data['digg_count'] = str(d['comment']['digg_count'])  # 点赞数量
                    dict_data['create_time'] = datetime.fromtimestamp(int(d['comment']['create_time'])) \
                        .strftime("%Y-%m-%d %H:%M:%S")  # 发布时间
                    with open("996.txt", "a+", encoding="utf-8") as fp:
                        fp.write("--------------------评论内容--------------------" + '\n')
                        fp.write("用户名称:" + dict_data['user_name'] + '\n')
                        fp.write("点赞数量:" + dict_data['digg_count'] + '\n')
                        fp.write("评论时间:" + dict_data['create_time'] + '\n')
                        fp.write("评论内容:" + dict_data['content'] + '\n')
                    print(f"=============获取【{dict_data['user_name']}】评论=============")
                if pagenum <= counts:
                    pagenum += 20
                    time.sleep(5)
                    self.get_comments(id, pagenum)

    # 删除HTML不需要标签
    def html_to_plain_text(self, html):
        text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)
        text = re.sub('<a\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)
        text = re.sub('<.*?>', '', text, flags=re.M | re.S)
        # 过滤标签，保留标签p
        # text = re.sub('(?!<(p).*?>|<(/p).*?>)<.*?>', '', text, flags=re.M | re.S)
        # text = re.sub(r'(\s*\n)+', '\n', text, flags=re.M | re.S)
        text = text.replace("\n", "")
        return unescape(text)

if __name__ == '__main__':
    toutiao = TouTiao()
    for p in range(0, 300, 10):
        toutiao.get_contents(p)
        print(f"=======第{p}条=========")
        time.sleep(3)

    # # 获取评论内容
    # toutiao.get_comments("6825530776637932040", 0)

    # 获取正文内容
    # toutiao.get_contents_1("6772468852472152590", "")